UK Police Crime Data¶
- This notebook provides a way to visualise crime data provided by the UK police forces
Motivation¶
- Looking for areas to rent/buy a house from the perspective of crime
- There is a crime map available on https://www.police.uk/metropolitan/E05009330/crime/
- The data was good and easy to visualise on their website but was designed for looking at one month at a time and one region
- We wanted to look at the data to compare different areas and felt that a longer period than one month to gauge crime rate
- There is also https://www.met.police.uk/sd/stats-and-data/met/crime-data-dashboard/ which is not street level
Imports¶
In [1]:
%matplotlib inline
import pandas as pd
import os
import getpass
import json
import seaborn
import matplotlib.pyplot as plt
from IPython.display import display
import geopandas as gpd
from shapely.geometry import Point
import plotly.graph_objs as go
from plotly.offline import init_notebook_mode, iplot
init_notebook_mode(connected=True)
def pythonise_string(input_string):
return input_string.lower().replace(' ', '_')
Mapbox Authentication¶
In [2]:
try:
with open('authentication.json') as f:
mapbox_access_token = json.load(f)['mapbox_access_token']
except:
mapbox_access_token = getpass.getpass()
User Parameters¶
In [3]:
data_path = r'D:\uk_police_crime_data\police_data'
region = 'metropolitan'
data_type = 'street'
start_date = pd.Timestamp(year=2014, month=12, day=1)
end_date = pd.Timestamp(year=2019, month=5, day=1)
map_data_path = r'D:\uk_police_crime_data\map_data\statistical-gis-boundaries-london\ESRI\London_Borough_Excluding_MHW.shp'
crs = {'init': 'epsg:4326'}
Raw Data¶
In [5]:
def get_data_file_path(date, data_path=data_path, region=region, data_type=data_type):
return os.path.join(data_path, f'{date:%Y-%m}', f'{date:%Y-%m}-{region}-{data_type}.csv')
raw_df = pd.concat([pd.read_csv(get_data_file_path(date)) for date in data_date_range], axis=0).reset_index(drop=True)
display(raw_df.head())
Clean Data¶
In [6]:
def clean_data(input_df):
df = input_df.copy()
df.columns = [pythonise_string(column) for column in df.columns]
df['date'] = pd.to_datetime(df['month'], format='%Y-%m')
df[['longitude', 'latitude']] = df[['longitude', 'latitude']].round(4)
df = df.loc[~(df['longitude'].isnull() | df['latitude'].isnull())]
drop_columns = ['month', 'falls_within', 'reported_by', 'last_outcome_category', 'context', 'lsoa_code', 'lsoa_name',
'crime_id']
df = df.drop(columns=drop_columns).reset_index(drop=True)
return df
clean_df = clean_data(raw_df)
display(clean_df['crime_type'].unique())
display(clean_df.head())
Geopandas¶
- This allows us to utilise geometric ideas such as points, polygons and SPATIAL JOINS!
In [7]:
gdf = clean_df.copy()
gdf['coordinate'] = list(zip(gdf['longitude'], gdf['latitude']))
gdf['coordinate'] = gdf['coordinate'].apply(Point)
gdf = gpd.GeoDataFrame(gdf, crs=crs, geometry='coordinate')
display(gdf.head())
display(gdf.crs)
In [8]:
map_df = gpd.read_file(map_data_path)
map_df.columns = [pythonise_string(column) for column in map_df.columns]
map_df = map_df.drop(columns=['gss_code', 'sub_2009', 'sub_2006', 'nonld_area', 'ons_inner'])
display(map_df.head())
display(map_df.crs)
In [9]:
map_df = map_df.to_crs({'proj': 'latlong', 'ellps': 'WGS84', 'datum': 'WGS84', 'no_defs': True})
# It is probably not a good thing to override the crs...
map_df.crs = crs
display(map_df.head())
display(map_df.crs)
connected_df = gpd.sjoin(map_df, gdf, how="left", op='contains').reset_index(drop=True)
Visualise using Matplotlib¶
In [10]:
fig, ax = plt.subplots(1, figsize=(10, 6))
map_df.plot(ax=ax)
gdf.loc[connected_df['index_right']].plot(ax=ax, marker='o', color='red', alpha=0.1, markersize=0.1)
Out[10]:
Visualise using Plotly and Mapbox¶
In [11]:
count_column_name = 'crime_type'
groupby_columns = ['longitude', 'latitude', 'crime_type']
remaining_column = ['location']
crime_df = clean_df.loc[connected_df['index_right']].groupby(groupby_columns)[remaining_column].count() \
.unstack('crime_type') \
.fillna(0) \
.astype('int64')
crime_df.columns = crime_df.columns.droplevel(0)
crime_df['Total'] = crime_df.sum(axis=1)
display(crime_df.head())
In [21]:
plot_df = crime_df[['Total']].rename(columns={'Total': 'count'})
plot_df['text'] = crime_df.apply(
lambda row: str({key: f'{value}<br>' for key, value in row.iteritems() if value != 0}),
axis=1,
).str.replace("'|, |{|}", '')
plot_df = plot_df.reset_index()
plot_df['percentile'] = plot_df['count'].rank(pct=True)
data = [
go.Scattermapbox(
lat=plot_df['latitude'],
lon=plot_df['longitude'],
mode='markers',
marker=go.scattermapbox.Marker(
size=7,
opacity=.5,
color=1-plot_df['percentile'],
colorscale='YlOrRd',
),
text=plot_df['text'],
)
]
layout = go.Layout(
autosize=True,
hovermode='closest',
mapbox=go.layout.Mapbox(
accesstoken=mapbox_access_token,
bearing=0,
style='streets',
center=go.layout.mapbox.Center(
lat=51.509865,
lon=-0.118092,
),
pitch=0,
zoom=8,
),
)
fig = go.Figure(data=data, layout=layout)
In [22]:
iplot(fig)